/* StatisticsSummary 
 * 
 * $Id: StatisticsSummary.java 6428 2009-08-04 01:22:52Z gojomo $$ 
 * 
 * Created on July 27, 2006
 * 
 * Copyright (C) 2009 Internet Archive.
 *
 * This file is part of the Heritrix web crawler (crawler.archive.org).
 *
 * Heritrix is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * any later version.
 *
 * Heritrix is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser Public License for more details.
 *
 * You should have received a copy of the GNU Lesser Public License
 * along with Heritrix; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
package org.archive.crawler.admin;

import java.io.File;
import java.io.FileReader;
import java.io.BufferedReader;
import java.io.IOException;
import java.util.Comparator;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.logging.Logger;



/**
 * This class provides descriptive statistics of a finished crawl job by
 * using the crawl report files generated by StatisticsTracker.  Any formatting
 * changes to the way StatisticsTracker writes to the summary crawl reports will
 * require changes to this class.
 * <p>
 * The following statistics are accessible from this class:
 * <ul>
 *   <li> Successfully downloaded documents per fetch status code
 *   <li> Successfully downloaded documents per document mime type
 *   <li> Amount of data per mime type
 *   <li> Successfully downloaded documents per host
 *   <li> Amount of data per host
 *   <li> Successfully downloaded documents per top-level domain name (TLD)
 *   <li> Disposition of all seeds 
 *   <li> Successfully downloaded documents per host per source
 * </ul>
 *
 * <p>TODO: Make it so summarizing is not done all in RAM so we avoid
 * OOME.
 *
 * @author Frank McCown
 *
 * @see org.archive.crawler.admin.StatisticsTracker
 */
public class StatisticsSummary {
    /**
     * Messages from the StatisticsSummary.
     */
    private final static Logger logger =
        Logger.getLogger(StatisticsSummary.class.getName());
    
    private boolean stats = true;
    
    /** Crawl job whose summary we want to view */
    private CrawlJob cjob;
        
    protected long totalDnsStatusCodeDocuments = 0;
    protected long totalStatusCodeDocuments = 0;
    protected long totalFileTypeDocuments = 0;
    protected long totalMimeTypeDocuments = 0;
    protected long totalDnsMimeTypeDocuments = 0;
    protected long totalDnsHostDocuments = 0;
    protected long totalHostDocuments = 0;
    protected long totalMimeSize = 0;
    protected long totalDnsMimeSize = 0;
    protected long totalHostSize = 0;
    protected long totalDnsHostSize = 0;
    protected long totalTldDocuments = 0;
    protected long totalTldSize = 0;
    protected long totalHosts = 0;
    
    protected String durationTime;
    protected String processedDocsPerSec;
    protected String bandwidthKbytesPerSec;
    protected String totalDataWritten;
    
    /** Keep track of the file types we see (mime type -> count) */
    protected Hashtable<String,AtomicLong> mimeTypeDistribution = new Hashtable<String,AtomicLong>();
    protected Hashtable<String,AtomicLong> mimeTypeBytes = new Hashtable<String,AtomicLong>();
    protected Hashtable<String,AtomicLong> mimeTypeDnsDistribution = new Hashtable<String,AtomicLong>();
    protected Hashtable<String,AtomicLong> mimeTypeDnsBytes = new Hashtable<String,AtomicLong>();
    
    /** Keep track of status codes */
    protected Hashtable<String,AtomicLong> statusCodeDistribution = new Hashtable<String,AtomicLong>();
    protected Hashtable<String,AtomicLong> dnsStatusCodeDistribution
     = new Hashtable<String,AtomicLong>();
    
    /** Keep track of hosts */
    protected Hashtable<String,AtomicLong> hostsDistribution = new Hashtable<String,AtomicLong>(); 
    protected Hashtable<String,AtomicLong> hostsBytes = new Hashtable<String,AtomicLong>(); 
    protected Hashtable<String,AtomicLong> hostsDnsDistribution = new Hashtable<String,AtomicLong>();
    protected Hashtable<String,AtomicLong> hostsDnsBytes = new Hashtable<String,AtomicLong>(); 

    /** Keep track of TLDs */
    protected Hashtable<String,AtomicLong> tldDistribution = new Hashtable<String,AtomicLong>();
    protected Hashtable<String,AtomicLong> tldBytes = new Hashtable<String,AtomicLong>();
    protected Hashtable<String,AtomicLong> tldHostDistribution = new Hashtable<String,AtomicLong>();

    /** Keep track of processed seeds */
    protected transient Map<String,SeedRecord> processedSeedsRecords 
     = new Hashtable<String,SeedRecord>();

    /**
     * Constructor
     * 
     * @param cjob
     * 				Completed crawl job
     */
    public StatisticsSummary(CrawlJob cjob) {
    	this.cjob = cjob;
    	
    	// Read all stats for this crawl job
    	this.stats = calculateStatusCodeDistribution();
    	if (calculateMimeTypeDistribution()) {
    		this.stats = true;
    	}
    	if (calculateHostsDistribution()) {
    		this.stats = true;
    	}
    	if (readCrawlReport()) {
    		this.stats = true;
    	}
    	if (readSeedReport()) {
    		this.stats = true;
    	}
    }
    
    
    /**
     * Increment a counter for a key in a given HashMap. Used for various
     * aggregate data.
     *
     * @param map The HashMap
     * @param key The key for the counter to be incremented, if it does not
     *               exist it will be added (set to 1).  If null it will
     *            increment the counter "unknown".
     */
    protected static void incrementMapCount(Map<String,AtomicLong> map, 
            String key) {
    	incrementMapCount(map,key,1);
    }

    /**
     * Increment a counter for a key in a given HashMap by an arbitrary amount.
     * Used for various aggregate data. The increment amount can be negative.
     *
     * @param map
     *            The HashMap
     * @param key
     *            The key for the counter to be incremented, if it does not
     *            exist it will be added (set to equal to
     *            <code>increment</code>).
     *            If null it will increment the counter "unknown".
     * @param increment
     *            The amount to increment counter related to the
     *            <code>key</code>.
     */
    protected static void incrementMapCount(Map<String,AtomicLong> map, 
            String key, long increment) {
        if (key == null) {
            key = "unknown";
        }
        // FIXME: for true safety this may need to use ConcurrenMap methods
        AtomicLong lw = map.get(key);
        if(lw == null) {
            map.put(key, new AtomicLong(increment));
        } else {
            lw.addAndGet(increment);
        }
    }
  
    /** Returns a HashMap that contains information about distributions of
     *  encountered mime types.  Key/value pairs represent
     *  mime type -> count.
     * <p>
     * <b>Note:</b> All the values are wrapped with a
     * {@link AtomicLong AtomicLong}
     * @return mimeTypeDistribution
     */
    public Hashtable getMimeDistribution() {
        return mimeTypeDistribution;
    }
    
    public long getTotalMimeTypeDocuments() {
       	return totalMimeTypeDocuments;
    }
    
    public long getTotalDnsMimeTypeDocuments() {
       	return totalDnsMimeTypeDocuments;
    }
    
    public long getTotalMimeSize() {
    	return totalMimeSize;
    }
    
    public long getTotalDnsMimeSize() {
    	return totalDnsMimeSize;
    }
   
    /**
     * Return a HashMap representing the distribution of HTTP status codes for
     * successfully fetched curis, as represented by a hashmap where key -&gt;
     * val represents (string)code -&gt; (integer)count.
     * 
     * <b>Note: </b> All the values are wrapped with a
     * {@link AtomicLong AtomicLong}
     * 
     * @return statusCodeDistribution
     */
    public Hashtable getStatusCodeDistribution() {    	
        return statusCodeDistribution;
    }
   
    /**
     * Return a HashMap representing the distribution of DNS status codes for
     * successfully fetched curis, as represented by a hashmap where key -&gt;
     * val represents (string)code -&gt; (integer)count.
     * 
     * <b>Note: </b> All the values are wrapped with a
     * {@link AtomicLong AtomicLong}
     * 
     * @return dnsStatusCodeDistribution
     */
    public Hashtable getDnsStatusCodeDistribution() {
    	return dnsStatusCodeDistribution;
    }
    
    public Hashtable getDnsMimeDistribution() {
        return mimeTypeDnsDistribution;
    }

    public long getTotalDnsStatusCodeDocuments() {
    	return totalDnsStatusCodeDocuments;
    }
    
    public long getTotalStatusCodeDocuments() {
    	return totalStatusCodeDocuments;
    }  
    
    public long getTotalHostDocuments() {
       	return totalHostDocuments;
    }
    
    public long getTotalDnsHostDocuments() {
       	return totalDnsHostDocuments;
    }
    
    public Hashtable getHostsDnsDistribution() {
    	return hostsDnsDistribution;
    }
    
    public long getTotalHostDnsDocuments() {
    	return totalDnsHostDocuments;
    }
    
    public long getTotalHostSize() {
    	return totalHostSize;
    }
    
    public long getTotalDnsHostSize() {
    	return totalDnsHostSize;
    }
    
    public Hashtable getTldDistribution() {
    	return tldDistribution;
    }
    
    public Hashtable getTldBytes() {
    	return tldBytes;
    }
    
    public long getTotalTldDocuments() {
    	return totalTldDocuments;
    }
    
    public long getTotalTldSize() {
    	return totalTldSize;
    }
    
    public Hashtable getTldHostDistribution() {
    	return tldHostDistribution;
    }
    
    public long getTotalHosts() {
    	return totalHosts;
    }
    
    public String getDurationTime() {
    	return durationTime;
    }
    
    public String getProcessedDocsPerSec() {
    	return processedDocsPerSec;
    }
    
    public String getBandwidthKbytesPerSec() {
    	return bandwidthKbytesPerSec;
    }
    
    public String getTotalDataWritten() {
    	return totalDataWritten;
    }

    /**
     * Sort the entries of the given HashMap in descending order by their
     * values, which must be <code>AtomicLong</code>s.
     * <p>
     * Elements are sorted by value from largest to smallest. Equal values are
     * sorted in an arbitrary, but consistent manner by their keys. Only items
     * with identical value and key are considered equal.
     *
     * If the passed-in map requires access to be synchronized, the caller
     * should ensure this synchronization. 
     * 
     * @param mapOfAtomicLongValues
     *            Assumes values are AtomicLongs.
     * @return a sorted set containing the same elements as the map.
     */
    public TreeMap<String,AtomicLong> getReverseSortedCopy(
            final Map<String,AtomicLong> mapOfAtomicLongValues) {
        TreeMap<String,AtomicLong> sortedMap = new TreeMap<String,AtomicLong>(
          new Comparator<String>() {
            public int compare(String e1, String e2) {
                long firstVal = mapOfAtomicLongValues.get(e1).get();
                long secondVal = mapOfAtomicLongValues.get(e2).get();
                if (firstVal < secondVal) {
                    return 1;
                }
                if (secondVal < firstVal) {
                    return -1;
                }
                // If the values are the same, sort by keys.
                return e1.compareTo(e2);
            }
        });
        try {
            sortedMap.putAll(mapOfAtomicLongValues);
        } catch (UnsupportedOperationException e) {
            for (String key: mapOfAtomicLongValues.keySet()) {
                sortedMap.put(key, mapOfAtomicLongValues.get(key));
            }
        }
        return sortedMap;
    }
     
    /**
     * Get the number of hosts with a particular TLD.
     * @param tld
     * 				top-level domain name
     * @return		Total crawled hosts
     */
    public long getHostsPerTld(String tld) {
    	AtomicLong lw = (AtomicLong)tldHostDistribution.get(tld);
    	return (lw == null ? 0 : lw.get());
    }
    
    /**
     * Read status code distribution from responsecode-report.txt.
     * DNS and HTTP status codes are separated when read.
     * @return True if we found some stats.
     */
    private boolean calculateStatusCodeDistribution() {
    	// Read from responsecode-report.txt
    	File f = new File(cjob.getDirectory(), "responsecode-report.txt");
    	if (!f.exists()) {
    		return false;
    	}
    	BufferedReader br = null;
    	try {
	    	FileReader reader = new FileReader(f);
	    	br = new BufferedReader(reader);
	    	String line = br.readLine();  // Ignore heading
	    	line = br.readLine();
	    	while (line != null) {  	  
	    	  // Get status code and # urls which are seperated by a space
	    	  
	    	  String[] items = line.split(" ");
	    	  if (items.length < 2) {
	    		  logger.log(Level.WARNING,
                          "Unexpected formatting on line [" + line + "]");
	    	  }
	    	  else {
	    		  // See if DNS or HTTP status code
	    		  if (items[0].length() < 3) {
	    			  // DNS status code
	    			  long total = Long.parseLong(items[1]);
	    			  dnsStatusCodeDistribution.put(items[0], 
	    					  new AtomicLong(total));
	    			  totalDnsStatusCodeDocuments += total;
	    		  }
	    		  else {
	    			  // HTTP status code
	    			  long total = Long.parseLong(items[1]);
	    			  statusCodeDistribution.put(items[0], 
	    					  new AtomicLong(total));
	    			  totalStatusCodeDocuments += total;
	    		  }
	    	  }
	    	  line = br.readLine();
	    	}
    	} catch (IOException e) {
    		logger.log(Level.SEVERE, "Unable to read " + f.getAbsolutePath(),
    			e);
    	} finally {
    		if (br != null) {
    			try {
					br.close();
				} catch (IOException e) {
					logger.log(Level.SEVERE,
						"Closing " + f.getAbsolutePath(), e);
				}
    		}
    	}
    	return true;
    }
    
    /**
     * Read MIME type data from mimetype-report.txt.
     * MIME type of text/dns is separated from other MIME types.
     * @return True if we found some stats.
     */
    private boolean calculateMimeTypeDistribution() {    	
    	File f = new File(cjob.getDirectory(), "mimetype-report.txt");
    	if (!f.exists()) {
    		return false;
    	}
    	BufferedReader br = null;
    	try {
	    	FileReader reader = new FileReader(f);
	    	br = new BufferedReader(reader);
	    	String line = br.readLine();  // Ignore heading
	    	line = br.readLine();
	    	while (line != null) {	    			    	  
	    		// Get num urls, num bytes, and MIME type (seperated by a space)
	    		// Example: 12 134279 text/html
  
	    		String[] items = line.split(" ");
	    		if (items.length < 3) {
	    			logger.log(Level.WARNING,
                            "Unexpected formatting on line [" + line + "]");
	    		}
	    		else {
	    			long total = Long.parseLong(items[0]);
	    			long bytes = Long.parseLong(items[1]);
	    			String mime = items[2];

	    			// Seperate DNS reconrds from HTTP
	    			if (mime.equalsIgnoreCase("text/dns")) {
	    				mimeTypeDnsDistribution.put(mime,
                                new AtomicLong(total));
	    				mimeTypeDnsBytes.put(mime, new AtomicLong(bytes));
	    				totalDnsMimeTypeDocuments += total;
	    				totalDnsMimeSize += bytes;
	    			}
	    			else {
	    				mimeTypeDistribution.put(mime, new AtomicLong(total));
	    				mimeTypeBytes.put(mime, new AtomicLong(bytes));
	    				totalMimeTypeDocuments += total;
	    				totalMimeSize += bytes;
	    			}
	    		}
	    		line = br.readLine();
	    	}
    	} catch (IOException e) {
    		logger.log(Level.SEVERE, "Reading " + f.getAbsolutePath(), e);
    	} finally {
    		if (br != null) {
    			try {
    				br.close();
    			} catch (IOException e) {
    				logger.log(Level.SEVERE,
    					"Closing " + f.getAbsolutePath(), e);
    			}
    		}
    	}
    	return true;
    }
    
    /**
     * Read number of URLs and total bytes for each host name from
     * hosts-report.txt.
     * Host name of "dns:" is separated from others.
     * @return true if stats found.
     */
    private boolean calculateHostsDistribution() {
    	File f = new File(cjob.getDirectory(), "hosts-report.txt");
    	if (!f.exists()) {
    		return false;
    	}
    	BufferedReader br = null;
    	try {
	    	FileReader reader = new FileReader(f);
	    	br = new BufferedReader(reader);
	    	String line = br.readLine();  // Ignore heading
	    	line = br.readLine();
	    	while (line != null) {    	  
	    		// Get num urls, num bytes, and host name (seperated by a space)
	    		// Example: 9 7468 www.blogger.com

	    		String[] items = line.split(" ");
	    		if (items.length < 3) {
	    			logger.log(Level.WARNING,
                            "Unexpected formatting on line [" + line + "]");
	    		}
	    		else {
	    			long total = Long.parseLong(items[0]);
	    			long bytes = Long.parseLong(items[1]);
	    			String host = items[2];

	    			// Seperate DNS reconrds from HTTP
	    			if (host.startsWith("dns:", 0)) {
	    				hostsDnsDistribution.put(host, new AtomicLong(total));
	    				hostsDnsBytes.put(host, new AtomicLong(bytes));
	    				totalDnsHostDocuments += total;
	    				totalDnsHostSize += bytes;
	    			}
	    			else {
	    				hostsDistribution.put(host, new AtomicLong(total));
	    				hostsBytes.put(host, new AtomicLong(bytes));
	    				totalHostDocuments += total;
	    				totalHostSize += bytes;

	    				// Count top level domain (TLD)
	    				String tld = host.substring(host.lastIndexOf('.')+1);
	    				incrementMapCount(tldDistribution, tld, total);   
	    				incrementMapCount(tldBytes, tld, bytes);
	    				incrementMapCount(tldHostDistribution, tld);
	    				totalTldDocuments += total;
	    				totalTldSize += bytes;

	    				totalHosts++;
	    			}
	    		}
	    		line = br.readLine();
	    	}
    	} catch (IOException e) {
    		logger.log(Level.SEVERE, "Reading " + f.getAbsolutePath(), e);
    	} finally {
    		if (br != null) {
    			try {
    				br.close();
    			} catch (IOException e) {
    				logger.log(Level.SEVERE,
    					"Closing " + f.getAbsolutePath(), e);
    			}
    		}
    	}
    	return true;
    }

    /**
     * Returns the accumulated number of bytes downloaded from a given host.
     * @param host name of the host
     * @return the accumulated number of bytes downloaded from a given host
     */
    public long getBytesPerHost(String host) { 
    	long bytes = -1;
    	
    	bytes = host != null && host.startsWith("dns:", 0) ? 
	    	((AtomicLong)hostsDnsBytes.get(host)).get() :
	    	((AtomicLong)hostsBytes.get(host)).get();	    
    	
    	return bytes;
    }
    
    /**
     * Returns the total number of bytes downloaded for a given TLD.
     * @param tld TLD
     * @return the total number of bytes downloaded for a given TLD
     */
    public long getBytesPerTld(String tld) {
    	AtomicLong lw = (AtomicLong)tldBytes.get(tld);
    	return (lw == null ? 0 : lw.get());
    }

    /**
     * Returns the accumulated number of bytes from files of a given file type.
     * @param filetype Filetype to check.
     * @return the accumulated number of bytes from files of a given mime type
     */
    public long getBytesPerMimeType(String filetype) {
    	long bytes = -1;
    	
    	if (filetype != null) {    	
	    	if (filetype.equals("text/dns")) {	    		
	    		bytes = mimeTypeDnsBytes.get(filetype) == null ? 0 :
	    			((AtomicLong)mimeTypeDnsBytes.get(filetype)).get();
	    	}
	    	else {
	    		bytes = mimeTypeBytes.get(filetype) == null ? 0 :
	    			((AtomicLong)mimeTypeBytes.get(filetype)).get();
	    	}
    	}
    	return bytes;
    }
    
    /**
     * Reads duration time, processed docs/sec, bandwidth, and total size
     * of crawl from crawl-report.txt.
     * @return true if stats found.
     */
    public boolean readCrawlReport() {
    	File f = new File(cjob.getDirectory(), "crawl-report.txt");
    	if (!f.exists()) {
    		return false;
    	}
    	BufferedReader br = null;
    	try {
	    	FileReader reader = new FileReader(f);
	    	br = new BufferedReader(reader);
	    	String line = br.readLine();  
	    	while (line != null) {
	    		if (line.startsWith("Duration Time")) {
	    			durationTime = line.substring(line.indexOf(':')+1);
	    		}
	    		else if (line.startsWith("Processed docs/sec")) {
	    			processedDocsPerSec = line.substring(line.indexOf(':')+1);
	    		}
	    		else if (line.startsWith("Bandwidth in Kbytes/sec")) {
	    			bandwidthKbytesPerSec = line.substring(line.indexOf(':')+1);
	    		}
	    		else if (line.startsWith("Total Raw Data Size in Bytes")) {
	    			totalDataWritten = line.substring(line.indexOf(':')+1);
	    		}

	    		line = br.readLine();
	    	}
    	}
    	catch (IOException e) {
    		logger.log(Level.SEVERE, "Reading " + f.getAbsolutePath(), e);		
    	} finally {
    		if (br != null) {
    			try {
					br.close();
				} catch (IOException e) {
					logger.log(Level.SEVERE,
					    "Failed close of " + f.getAbsolutePath(), e);
				}
    		}
    	}
    	return true;
    }
  
    /**
     * Returns sorted Iterator of seeds records based on status code.
     * @return sorted Iterator of seeds records
     */
    public Iterator<SeedRecord> getSeedRecordsSortedByStatusCode() {
        TreeSet<SeedRecord> sortedSet = new TreeSet<SeedRecord>(
          new Comparator<SeedRecord>() {
            public int compare(SeedRecord sr1, SeedRecord sr2) {
                int code1 = sr1.getStatusCode();
                int code2 = sr2.getStatusCode();
                if (code1 == code2) {
                    // If the values are equal, sort by URIs.
                    return sr1.getUri().compareTo(sr2.getUri());
                }
                // mirror and shift the nubmer line so as to
                // place zero at the beginning, then all negatives 
                // in order of ascending absolute value, then all 
                // positives descending
                code1 = -code1 - Integer.MAX_VALUE;
                code2 = -code2 - Integer.MAX_VALUE;
                
                return new Integer(code1).compareTo(new Integer(code2));
            }
        });
        for (SeedRecord sr: processedSeedsRecords.values()) {
            sortedSet.add(sr);
        }
        
        return sortedSet.iterator();
    }
    
    /**
     * Reads seed data from seeds-report.txt.
     * @return True if stats found.
     */
    private boolean readSeedReport() {
    	File f = new File(cjob.getDirectory(), "seeds-report.txt");
    	if (!f.exists()) {
    		return false;
    	}
    	BufferedReader br = null;
    	try {
	    	FileReader reader = new FileReader(f);
	    	br = new BufferedReader(reader);
	    	
	    	// Ignore heading: [code] [status] [seed] [redirect]
	    	String line = br.readLine();  
	    	line = br.readLine();
	    	while (line != null) {
	    		// Example lines:
	    		// 302 CRAWLED http://www.ashlandcitytimes.com/ http://www.ashlandcitytimes.com/apps/pbcs.dll/section?Category=MTCN01
	    		// 200 CRAWLED http://noleeo.com/

	    		String[] items = line.split(" ");

	    		if (items.length < 3) {
	    			logger.log(Level.WARNING,
                            "Unexpected formatting on line [" + line + "]");
	    		}
	    		else {
	    			String statusCode = items[0];
	    			String crawlStatus = items[1];
	    			String seed = items[2];
	    			String redirect = items.length > 3 ? items[3] : null;

	    			// All values should be CRAWLED or NOTCRAWLED
	    			if (crawlStatus.equals("CRAWLED")) {
	    				crawlStatus =org.archive.crawler.framework.StatisticsTracking.SEED_DISPOSITION_SUCCESS;	    		  
	    			}
	    			else {
	    				crawlStatus = org.archive.crawler.framework.StatisticsTracking.SEED_DISPOSITION_FAILURE;
	    			}
	    			SeedRecord sr = new SeedRecord(seed, crawlStatus, 
	    					Integer.parseInt(statusCode), redirect);
	    			processedSeedsRecords.put(seed, sr);
	    		}

	    		line = br.readLine();
	    	}
    	} catch (IOException e) {
    		logger.log(Level.SEVERE, "Reading " + f.getAbsolutePath(), e);   		
    	} finally {
    		if (br != null) {
    			try {
					br.close();
				} catch (IOException e) {
					logger.log(Level.SEVERE,
						"Closing " + f.getAbsolutePath(), e);
				}
    		}
    	}
    	return true;
    }
        
    /**
     * Return a copy of the hosts distribution in reverse-sorted
     * (largest first) order.
     *  
     * @return SortedMap of hosts distribution
     */
    public SortedMap getReverseSortedHostsDistribution() {
        return getReverseSortedCopy(hostsDistribution);  
    }    
    
    /**
     * @return True if we compiled stats, false if none to compile (e.g.
     * there are no reports files on disk).
     */
    public boolean isStats() {
    	return this.stats;
    }
}
